package com.sbtr.business.feature.service.impl;

import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.bean.copier.BeanCopier;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.json.JSONArray;
import cn.hutool.json.JSONObject;
import cn.hutool.json.JSONUtil;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.sbtr.business.feature.dto.*;
import com.sbtr.business.feature.entity.*;
import com.sbtr.business.feature.mapper.*;
import com.sbtr.business.feature.service.FeDatafieldService;
import com.sbtr.business.feature.service.FeDatafileService;
import com.sbtr.business.feature.service.FeOperatorService;
import com.sbtr.business.feature.service.FePcaRuleService;
import com.sbtr.business.system.dto.DictionaryTreeDto;
import com.sbtr.business.system.entity.Dictionary;
import com.sbtr.business.system.entity.DictionaryParam;
import com.sbtr.business.system.mapper.DictionaryMapper;
import com.sbtr.business.system.service.DictionaryService;
import com.sbtr.business.utils.OperatorUtil;
import com.sbtr.common.PageDto;
import com.sbtr.common.calculate.CalculateReflectUtil;
import com.sbtr.common.calculate.CalculatorUtils;
import com.sbtr.common.calculate.DataExplorationUtils;
import com.sbtr.common.enums.CalculateEnum;
import com.sbtr.common.enums.DataTypeEnum;
import com.sbtr.common.enums.OperatorTypeEnum;
import com.sbtr.common.util.DataObjectConvertUtil;
import com.sbtr.exception.BDException;
import com.sbtr.util.PageDtoUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;

import javax.annotation.PostConstruct;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.stream.Collectors;

/**
 * 特征算子字段列表 Service接口实现类
 *
 * @author dgl
 * @since 2023-03-27
 */
@Slf4j
@Transactional
@Service
public class FeOperatorServiceImpl extends ServiceImpl<FeOperatorMapper, FeOperator> implements FeOperatorService {

    @Autowired
    FeOperatorMapper feOperatorMapper;
    @Autowired
    FeDatafileService feDatafileService;
    @Autowired
    FeDatafileMapper feDatafileMapper;
    @Autowired
    FeDatafieldMapper feDatafieldMapper;
    @Autowired
    FeDatafieldService feDatafieldService;
    @Autowired
    FeFilterRuleMapper feFilterRuleMapper;//过滤
    @Autowired
    FeMissingRuleMapper feMissingRuleMapper;//缺失
    @Autowired
    FeDuplicateRuleMapper feDuplicateRuleMapper;//重复
    @Autowired
    FeAttributecreateRuleMapper feAttributecreateRuleMapper;//生成
    @Autowired
    FeStandardRuleMapper feStandardRuleMapper;//数据标准化
    @Autowired
    FePcaRuleMapper fePcaRuleMapper;//主成分分析
    @Autowired
    OperatorUtil operatorUtil;//链路数据
    @Autowired
    CalculateReflectUtil calculateReflectUtil;//字典对应函数工具
    @Autowired
    FePcaRuleService fePcaRuleService; // PCA服务执行
    @Autowired
    DictionaryService dictionaryService;//字典
    // 指定PCA所属服务器IP及端口
    @Value("${train-server-ip-id}")
    private String trainServerIpId;

    @Override
    public PageDto<FeOperator> pageList(FeOperatorDto feOperatorDto) {

        IPage<FeOperator> page = new Page<>();
        page.setSize(feOperatorDto.getPageSize());
        page.setCurrent(feOperatorDto.getPage());
        page = feOperatorMapper.selectPages(page, BeanUtil.beanToMap(feOperatorDto));
        PageDto<FeOperator> r = PageDtoUtils.getPageDtoByIPage(page, FeOperator.class);

        return r;
    }

    @Override
    public List<FeOperatorTreeDto> getTree(Long canvasId) {
        Map<String, Object> paramsMap = new HashMap<>();
        paramsMap.put("canvasId", canvasId);
        List<FeOperatorTreeDto> list = feOperatorMapper.selectTree(paramsMap);
        List<FeOperatorTreeDto> result = new ArrayList<>();
        List<FeOperatorTreeDto> roots = new ArrayList<>();
        for (FeOperatorTreeDto feOperatorTreeDto : list) {
            if (feOperatorTreeDto.getPid() == 0) {
                roots.add(feOperatorTreeDto);
            }
        }
        if (CollectionUtils.isNotEmpty(list)) {
            for (FeOperatorTreeDto root : roots) {
                result.add(returnNote(root, list));
            }
        }
        return result;
    }

    //递归树结构算子
    private FeOperatorTreeDto returnNote(FeOperatorTreeDto root, List<FeOperatorTreeDto> list) {
        List<FeOperatorTreeDto> leave = new ArrayList<>();
        root.setChildren(new ArrayList<>());
        for (FeOperatorTreeDto feOperatorTreeDto : list) {
            if (feOperatorTreeDto.getPid().equals(root.getId())) {
                root.getChildren().add(feOperatorTreeDto);
            } else {
                leave.add(feOperatorTreeDto);
            }
        }
        if (CollectionUtils.isNotEmpty(root.getChildren())) {
            for (FeOperatorTreeDto children : root.getChildren()) {
                returnNote(children, leave);
            }
        }

        return root;
    }

    @Override
    public List<LinkedHashMap<String, String>> exec(FeOperatorExecDto feOperatorExecDto) {

        FeOperator feOperator = feOperatorMapper.selectById(feOperatorExecDto.getId());
        //字段id，字段名
        LinkedHashMap<Long, String> datafieldIds = new LinkedHashMap<>();
        List<LinkedHashMap<String, String>> list = new ArrayList<>();
        if (feOperator.getPid() != null && feOperator.getPid() != 0) {
            List<LinkedHashMap<String, String>> d = operatorUtil.getData(feOperator.getPid().toString());
            if (CollectionUtils.isNotEmpty(d)) {
                d.forEach(r -> {//地址一直传递，大坑
                    list.add(new LinkedHashMap<>(r));
                });
            }
        }
        List<LinkedHashMap<String, String>> data;
        if (CollectionUtils.isEmpty(list)) {
            QueryWrapper<FeDatafile> queryWrapper = new QueryWrapper<>();
            queryWrapper.eq("processing_id", feOperator.getPid());
            List<FeDatafile> feDatafiles = feDatafileMapper.selectList(queryWrapper);
            if (feDatafiles.size() != 1) {
                return new ArrayList<>();
//                throw new BDException("不存在执行文件或执行文件大于1");
            }
            //查询文件字段
            Map<String, Object> params = new HashMap<>();
            params.put("datafile_id", feDatafiles.get(0).getId());
            Collection<FeDatafield> feDatafields = feDatafieldMapper.selectByMap(params);//文件字段
            for (FeDatafield feDatafield : feDatafields) {
                datafieldIds.put(feDatafield.getId(), feDatafield.getName());
            }
            data = feDatafileService.analyze(feDatafiles.get(0).getId()).getListMap();
        } else {
            data = list;
            LinkedHashMap<Long, String> map = new LinkedHashMap<>();
            LinkedHashMap<Long, String> t = operatorUtil.getTitle(feOperator.getPid().toString());
            if (t != null) {
                map.putAll(t);
            }
            datafieldIds = map;
        }


        //读取文件数据
        List<LinkedHashMap<String, String>> result = new ArrayList<>();

        //处理类型
        if (feOperatorExecDto.getOperatorType().equals(OperatorTypeEnum.filter.getCode())) {
            //过滤
            result = filter(data, datafieldIds, feOperatorExecDto);
        } else if (feOperatorExecDto.getOperatorType().equals(OperatorTypeEnum.missing.getCode())) {
            //缺失填充
            result = missing(data, datafieldIds, feOperatorExecDto);
        } else if (feOperatorExecDto.getOperatorType().equals(OperatorTypeEnum.duplicate.getCode())) {
            //重复去重
            result = duplicate(data, datafieldIds, feOperatorExecDto);
        } else if (feOperatorExecDto.getOperatorType().equals(OperatorTypeEnum.attributecreate.getCode())) {
            //属性生成
            result = attributecreate(data, datafieldIds, feOperatorExecDto);
        } else if (feOperatorExecDto.getOperatorType().equals(OperatorTypeEnum.standard.getCode())) {
            //数据标准化
            result = standard(data, datafieldIds, feOperatorExecDto);
        } else if (feOperatorExecDto.getOperatorType().equals(OperatorTypeEnum.pca.getCode())) {
            //主成分分析
            result = pca(data, datafieldIds, feOperatorExecDto);
        }


        //装载结果
        operatorUtil.putData(feOperatorExecDto.getId().toString(), result);
        operatorUtil.putTitle(feOperatorExecDto.getId().toString(), datafieldIds);

        return result;
    }

    @Override
    public Object execExplore(FeOperatorExecExploreDto feOperatorExecExploreDto) throws InvocationTargetException, IllegalAccessException {
        List<Method> method1 = new ArrayList<>();
        for (String name : feOperatorExecExploreDto.getMethod()) {
            for (Method method : DataExplorationUtils.class.getMethods()) {
                if (method.getName().equals(name)) {
                    method1.add(method);
                }
            }
        }
        if (method1.size() == 0) {
            return null;
        }
        FeOperator feOperator = feOperatorMapper.selectById(feOperatorExecExploreDto.getId());

        List<LinkedHashMap<String, String>> list = operatorUtil.getSelectData(feOperator.getPid().toString(), feOperatorExecExploreDto.getField());

        LinkedHashMap<String, List<String>> data = new LinkedHashMap<>();

        list.forEach(r -> {
            for (Map.Entry<String, String> entry : r.entrySet()) {
                List<String> list1 = data.get(entry.getKey());
                if (CollectionUtils.isEmpty(list1)) {
                    list1 = new ArrayList<>();
                }
                list1.add(entry.getValue());
                data.put(entry.getKey(), list1);
            }
        });

        if(data.size()==0){
            throw new BDException("请先运行算子后再探索");
        }
        DataExplorationUtils dataExplorationUtils = new DataExplorationUtils();
        List<List<Object>> r = new ArrayList<>();
        //添加标题
        List<DictionaryTreeDto> dictionaryTreeDtos = dictionaryService.treeList("data_explore", new Dictionary());
        List<Object> o = new ArrayList<>();
        o.add("");
        dir:
        for (String name : feOperatorExecExploreDto.getMethod()) {
            for (DictionaryParam d : dictionaryTreeDtos.get(0).getChildrenParams()) {
                if (d.getOptCode().equals(name)) {
                    o.add(d.getOptName());
                    continue dir;
                }
            }
        }
        r.add(o);

        //添加内容
        for (String feild : feOperatorExecExploreDto.getField()) {
            List<Object> list1 = new ArrayList<>();
            list1.add(feild);
            line:
            for (String name : feOperatorExecExploreDto.getMethod()) {
                for (Method method : DataExplorationUtils.class.getMethods()) {
                    if (method.getName().equals(name)) {
                        Object o2 = DataObjectConvertUtil.convert(data.get(feild), DataTypeEnum.FLOAT.getCode());
                        Object r2 = method.invoke(dataExplorationUtils, o2);
                        list1.add(r2);
                        continue line;
                    }
                }
            }
            r.addAll(Collections.singleton(list1));
        }

        return r;
    }

    //过滤数据
    private List<LinkedHashMap<String, String>> filter(List<LinkedHashMap<String, String>> data, Map<Long, String> datafieldIds, FeOperatorExecDto feOperatorExecDto) {
        QueryWrapper<FeFilterRule> filterWrapper = new QueryWrapper<>();
        filterWrapper.eq("processing_id", feOperatorExecDto.getId());
        filterWrapper.in("datafield_id", datafieldIds.keySet());
        Collection<FeFilterRule> feFilterRules = feFilterRuleMapper.selectList(filterWrapper);//过滤
        //原字段名，过滤对比对象
        Map<String, FeFilterRule> filterParams = new HashMap<>();//需要过滤的条件对象
        for (FeFilterRule feFilterRule : feFilterRules) {
            filterParams.put(datafieldIds.get(feFilterRule.getDatafieldId()), feFilterRule);
        }

        List<LinkedHashMap<String, String>> result = new ArrayList<>();
        line:
        for (LinkedHashMap<String, String> param : data) {//行迭代
            boolean accord = false; //and条件，全部符合才加
            for (Map.Entry<String, FeFilterRule> filterParam : filterParams.entrySet()) {//列迭代，部分过滤字段
                String s = param.get(filterParam.getKey());
                if (StringUtils.isBlank(s)) continue;//过滤空的
                accord = true;//只要进来了说明不全为空;
                try {
                    //指定转换类型
                    Object o = convert(s, filterParam.getValue().getType());
                    //对比
                    if (StringUtils.isNotBlank(filterParam.getValue().getValue())) {
                        //对比对象转换
                        Object o2 = convert(filterParam.getValue().getValue(), filterParam.getValue().getConditionsType());
                        boolean pass = (boolean) calculateReflectUtil.exec(filterParam.getValue().getConditions(), o, o2);
                        if (!pass) {
                            continue line;//只要其中一个不符合，当前行不要
                        }
                    } else {//过滤
                        boolean pass = (boolean) calculateReflectUtil.exec(filterParam.getValue().getConditions(), o);
                        if (!pass) {
                            continue line;//只要其中一个不符合，当前行不要
                        }
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                    throw new BDException("解析失败，请判断数据格式是否正确");
                }
            }
            //不被跳过说明全部过滤条件符合
            if (accord || filterParams.size() == 0) {
                result.add(param);
            }
        }
        return result;
    }

    //填充数据
    private List<LinkedHashMap<String, String>> missing(List<LinkedHashMap<String, String>> data, Map<Long, String> datafieldIds, FeOperatorExecDto feOperatorExecDto) {
        QueryWrapper<FeMissingRule> missingWrapper = new QueryWrapper<>();
        missingWrapper.eq("processing_id", feOperatorExecDto.getId());
        missingWrapper.in("datafield_id", datafieldIds.keySet());
        Collection<FeMissingRule> feMissingRules = feMissingRuleMapper.selectList(missingWrapper);//缺失

        //原字段名，填充对比对象
        Map<String, FeMissingRule> feMissingParams = new HashMap<>();//需要填充的条件对象
        Map<String, FeMissingRule> missCustomParams = new HashMap<>();//需要填充的自定义参数对象
        for (FeMissingRule feMissingRule : feMissingRules) {
            if (feMissingRule.getMissingType().equals(CalculateEnum.custom.getCode())) {
                missCustomParams.put(datafieldIds.get(feMissingRule.getDatafieldId()), feMissingRule);
            } else {
                feMissingParams.put(datafieldIds.get(feMissingRule.getDatafieldId()), feMissingRule);
            }
        }

        //字段名，对应字段的列值
        Map<String, List<String>> params = new HashMap<>();
        for (Map<String, String> param : data) {//每个字段填充方式只选一种，自定义，统计
            //迭代填充空白
            for (Map.Entry<String, FeMissingRule> missingParam : missCustomParams.entrySet()) {
                String s = param.get(missingParam.getKey());//键肯定存在
                if (StringUtils.isNotBlank(s)) continue;//过滤不为空的
                param.put(missingParam.getKey(), missingParam.getValue().getValue());//填充自定义缺失值
            }
            //迭代数据装载
            for (Map.Entry<String, FeMissingRule> missingParam : feMissingParams.entrySet()) {
                String s = param.get(missingParam.getKey());
                if (StringUtils.isBlank(s)) continue;//待填充对象跳过
                List<String> list = params.get(missingParam.getKey());
                if (list == null) {
                    list = new ArrayList<>();
                }
                list.add(s);
                params.put(missingParam.getKey(), list);
            }
        }

        if (params.size() != 0) {//不为空说明填充方式为统计填充,否则为自定义填充
            //迭代统计填充
            for (Map<String, String> param : data) {
                for (Map.Entry<String, FeMissingRule> missingParam : feMissingParams.entrySet()) {
                    String s = param.get(missingParam.getKey());
                    if (StringUtils.isNotBlank(s)) continue;//存在值的跳过
                    //先获取对应字段的所有列数据
                    List<String> list = params.get(missingParam.getKey());
                    if (list == null || list.size() == 0) continue;//全列为空的跳过
                    try {
                        Object p = convert(list, missingParam.getValue().getType());
                        if (p == null) {
                            log.error("转换数组里类型失败");
                            continue;
                        }
                        String r;
                        if (CalculateEnum.custom.getCode().equals(missingParam.getValue().getMissing())) {
                            param.put(missingParam.getKey(), missingParam.getValue().getValue());
                            continue;
                        } else {
                            r = String.valueOf(calculateReflectUtil.exec(missingParam.getValue().getMissing(), p));
                        }
                        param.put(missingParam.getKey(), r);
                    } catch (Exception e) {
                        e.printStackTrace();
                        throw new BDException("解析失败，请判断数据格式是否正确");
                    }
                }
            }
        }
        return data;
    }

    //重复数据处理
    private List<LinkedHashMap<String, String>> duplicate(List<LinkedHashMap<String, String>> data, Map<Long, String> datafieldIds, FeOperatorExecDto feOperatorExecDto) {
        QueryWrapper<FeDuplicateRule> duplicateWrapper = new QueryWrapper<>();
        duplicateWrapper.eq("processing_id", feOperatorExecDto.getId());
        duplicateWrapper.in("datafield_id", datafieldIds.keySet());
        Collection<FeDuplicateRule> feDuplicateRules = feDuplicateRuleMapper.selectList(duplicateWrapper);//重复
        if (CollectionUtils.isEmpty(feDuplicateRules)) {
            return data;
        }
        //原字段名，重复对比对象
        Map<String, FeDuplicateRule> feDuplicateParams = new LinkedHashMap<>();//需要过滤的条件对象
        for (FeDuplicateRule feDuplicateRule : feDuplicateRules) {
            feDuplicateParams.put(datafieldIds.get(feDuplicateRule.getDatafieldId()), feDuplicateRule);
        }

        //过滤对象,一行
        Map<String, LinkedHashMap<String, String>> listMap = new LinkedHashMap<>();
        for (LinkedHashMap<String, String> param : data) {
            StringBuilder keys = new StringBuilder();
            for (Map.Entry<String, FeDuplicateRule> feDuplicateParam : feDuplicateParams.entrySet()) {//需顺序一致
                String s = param.get(feDuplicateParam.getKey());
                if (StringUtils.isBlank(s)) continue;
                keys.append(s).append("-");
            }
            if (StringUtils.isNotBlank(keys)) {
                String keys2 = keys.substring(0, keys.length() - 1);
                listMap.put(keys2, param);
            }

        }
        List<LinkedHashMap<String, String>> list = new ArrayList<>();
        if (listMap.size() != 0) {
            for (Map.Entry<String, LinkedHashMap<String, String>> mapEntry : listMap.entrySet()) {
                list.add(mapEntry.getValue());
            }
            return list;
        } else {
            return data;
        }
    }

    //属性生成
    private List<LinkedHashMap<String, String>> attributecreate(List<LinkedHashMap<String, String>> data, Map<Long, String> datafieldIds, FeOperatorExecDto feOperatorExecDto) {
        //因为多个结果可能存在部分多余
        QueryWrapper<FeAttributecreateRule> attributecreateWrapper = new QueryWrapper<>();
        attributecreateWrapper.eq("processing_id", feOperatorExecDto.getId());
        Collection<FeAttributecreateRule> feAttributecreateRules = feAttributecreateRuleMapper.selectList(attributecreateWrapper);//生成

        //添加字段表信息
        QueryWrapper<FeDatafield> feDatafieldQueryWrapper2 = new QueryWrapper<>();
        feDatafieldQueryWrapper2.in("id", datafieldIds.keySet());
        List<FeDatafield> oldFields2 = feDatafieldMapper.selectList(feDatafieldQueryWrapper2);
        List<FeDatafield> oldFields = new ArrayList<>();
        List<FeDatafield> newFields = new ArrayList<>();

        if (oldFields2.size() != 0) {
            QueryWrapper<FeDatafield> feDatafieldQueryWrapper = new QueryWrapper<>();
            feDatafieldQueryWrapper.in("datafile_id", oldFields2.get(0).getDatafileId());
            oldFields = feDatafieldMapper.selectList(feDatafieldQueryWrapper);
        }
        line:
        for (FeAttributecreateRule feAttributecreateRule : feAttributecreateRules) {
            if (StringUtils.isBlank(feAttributecreateRule.getNewfield())) continue;
            for (FeDatafield old : oldFields) {
                if (old.getName().equals(feAttributecreateRule.getNewfield())) {
                    continue line;
                }
            }
            //不存在就添加到对应字段信息表
            FeDatafield feDatafield = new FeDatafield();
            feDatafield.setDatafileId(oldFields.get(0).getDatafileId());//任意取一条 都一样
            feDatafield.setType(DataTypeEnum.FLOAT.getCode());
            feDatafield.setName(feAttributecreateRule.getNewfield());
            feDatafield.setCreateTime(new Date());
            feDatafield.setSource(2);
            newFields.add(feDatafield);

        }
        feDatafieldService.saveBatch(newFields);

        line:
        for (Map<String, String> param : data) {
            for (FeAttributecreateRule feAttributecreateRule : feAttributecreateRules) {
                try {
                    String formula = feAttributecreateRule.getFormula();

                    Map<String, Double> param2 = new HashMap<>();
                    for (String fieldname : datafieldIds.values()) {
                        int n = formula.indexOf(fieldname);
                        String s = param.get(fieldname);
                        if (n == -1) continue;
                        if (StringUtils.isBlank(s)) continue;
                        //转换格式
                        param2.put(fieldname, Double.valueOf(s));
                    }
                    Double r;

                    //内置函数
                    r = CalculatorUtils.evaluate(feAttributecreateRule.getFormula(), param2);

                    //生成值
                    if (StringUtils.isNotBlank(feAttributecreateRule.getNewfield())) {
                        param.put(feAttributecreateRule.getNewfield(), r.toString());
                    }

                } catch (Exception e) {
                    e.printStackTrace();
                    throw new BDException("解析失败，请判断数据格式是否正确");
                }
            }
        }

        return data;
    }

    //数据标准化
    private List<LinkedHashMap<String, String>> standard(List<LinkedHashMap<String, String>> data, Map<Long, String> datafieldIds, FeOperatorExecDto feOperatorExecDto) {
        //因为多个结果可能存在部分多余
        QueryWrapper<FeStandardRule> standardRuleQueryWrapper = new QueryWrapper<>();
        standardRuleQueryWrapper.eq("processing_id", feOperatorExecDto.getId());
        standardRuleQueryWrapper.in("datafield_id", datafieldIds.keySet());
        Collection<FeStandardRule> feStandardRules = feStandardRuleMapper.selectList(standardRuleQueryWrapper);//生成

        //添加字段表信息
        QueryWrapper<FeDatafield> feDatafieldQueryWrapper2 = new QueryWrapper<>();
        feDatafieldQueryWrapper2.in("id", datafieldIds.keySet());
        List<FeDatafield> oldFields2 = feDatafieldMapper.selectList(feDatafieldQueryWrapper2);
        List<FeDatafield> oldFields = new ArrayList<>();
        List<FeDatafield> newFields = new ArrayList<>();

        if (oldFields2.size() != 0) {
            QueryWrapper<FeDatafield> feDatafieldQueryWrapper = new QueryWrapper<>();
            feDatafieldQueryWrapper.in("datafile_id", oldFields2.get(0).getDatafileId());
            oldFields = feDatafieldMapper.selectList(feDatafieldQueryWrapper);
        }

        line:
        for (FeStandardRule feStandardRule : feStandardRules) {
            if (StringUtils.isBlank(feStandardRule.getFieldname2())) continue;
            for (FeDatafield old : oldFields) {
                if (old.getName().equals(feStandardRule.getFieldname2())) {
                    continue line;
                }
            }
            //不存在就添加到对应字段信息表
            FeDatafield feDatafield = new FeDatafield();
            feDatafield.setDatafileId(oldFields.get(0).getDatafileId());//任意取一条 都一样
            feDatafield.setType(DataTypeEnum.FLOAT.getCode());
            feDatafield.setName(feStandardRule.getFieldname2());
            feDatafield.setCreateTime(new Date());
            feDatafield.setSource(2);
            newFields.add(feDatafield);

        }
        feDatafieldService.saveBatch(newFields);

        //原字段名，标准化对比对象
        Map<String, FeStandardRule> feStandardRuleParams = new LinkedHashMap<>();//需要过滤的条件对象
        for (FeStandardRule feStandardRule : feStandardRules) {
            feStandardRuleParams.put(datafieldIds.get(feStandardRule.getDatafieldId()), feStandardRule);
        }

        //字段名，对应字段的列值
        Map<String, List<String>> params = new HashMap<>();
        for (Map<String, String> param : data) {//每个字段填充方式只选一种，自定义，统计
            //迭代数据装载
            for (Map.Entry<String, FeStandardRule> standarParam : feStandardRuleParams.entrySet()) {
                String s = param.get(standarParam.getKey());
                if (StringUtils.isBlank(s)) s = "0";
                List<String> list = params.get(standarParam.getKey());
                if (list == null) {
                    list = new ArrayList<>();
                }
                list.add(s);
                params.put(standarParam.getKey(), list);
            }
        }

        Map<String, double[]> newData = new HashMap<>();
        //迭代每项数据标准化操作获取数组
        for (Map.Entry<String, FeStandardRule> feStandardRuleEntry : feStandardRuleParams.entrySet()) {
            try {
                List<String> list = params.get(feStandardRuleEntry.getKey());
                if (list == null || list.size() == 0) continue;//全列为空的跳过
                try {
                    Object p = convert(list, DataTypeEnum.FLOAT2.getCode());
                    if (p == null) {
                        log.error("转换数组里类型失败");
                        continue;
                    }
                    double[] r = (double[]) (calculateReflectUtil.exec(feStandardRuleEntry.getValue().getStandard(), p));
                    newData.put(feStandardRuleEntry.getValue().getFieldname2(), r);
                } catch (Exception e) {
                    e.printStackTrace();
                    throw new BDException("解析失败，请判断数据格式是否正确");
                }

            } catch (Exception e) {
                e.printStackTrace();
                throw new BDException("解析失败，请判断数据格式是否正确");
            }
        }
        //设置数据标准化后的列
        for (int i = 0; i < data.size(); i++) {
            for (Map.Entry<String, double[]> entry : newData.entrySet()) {
                data.get(i).put(entry.getKey(), String.valueOf(entry.getValue()[i]));
            }
        }

        return data;
    }


    //主成分分析
    private List<LinkedHashMap<String, String>> pca(List<LinkedHashMap<String, String>> data, Map<Long, String> datafieldIds, FeOperatorExecDto feOperatorExecDto) {
        //因为多个结果可能存在部分多余
        QueryWrapper<FePcaRule> pcaRuleQueryWrapper = new QueryWrapper<>();
        pcaRuleQueryWrapper.eq("processing_id", feOperatorExecDto.getId());
        pcaRuleQueryWrapper.in("datafield_id", datafieldIds.keySet());
        List<FePcaRule> fePcaRuleList = fePcaRuleMapper.selectList(pcaRuleQueryWrapper);//pca

        //去标题转换数据格式
        List<LinkedHashMap<String, String>> operatorData = new ArrayList<>();

        List<Integer> overLine = new ArrayList<>();
        line:
        for (int i = 0; i < data.size(); i++) {
            LinkedHashMap<String, String> param2 = new LinkedHashMap<>();
            //迭代数据装载
            for (FePcaRule fePcaRule : fePcaRuleList) {
                String s = data.get(i).get(fePcaRule.getFieldname());
                if (StringUtils.isBlank(s)) continue line;//跳过
                param2.put(fePcaRule.getFieldname(), s);
            }
            operatorData.add(param2);
        }

        line:
        for (LinkedHashMap<String, String> param : data) {//筛选需要的分析的
            LinkedHashMap<String, String> param2 = new LinkedHashMap<>();
            //迭代数据装载
            for (FePcaRule fePcaRule : fePcaRuleList) {
                String s = param.get(fePcaRule.getFieldname());
                if (StringUtils.isBlank(s)) s = "0";//跳过
                param2.put(fePcaRule.getFieldname(), s);
            }
            operatorData.add(param2);

        }

        if (CollectionUtils.isNotEmpty(fePcaRuleList)) {
            List<String> titles = new ArrayList<>();
            titles.addAll(new ArrayList<>(operatorData.get(0).keySet()));
            List<List<String>> list = operatorUtil.getArrayData(operatorData, false, titles);

//            List<List<Double>> list2= new ArrayList<>();
//            for (List<String> s :list) {
//                List<Double> list1 = s.stream().map(Double::parseDouble).collect(Collectors.toList());
//                list2.add(list1);
//            }

            String cum_var = fePcaRuleList.get(0).getRule();     //因为一个算子只能一个累计贡献度

            String pcaResultStr = fePcaRuleService.exePythonPCA(list, cum_var, trainServerIpId);

            FePcaRuleResultDto fePcaRuleResultDto = JSONUtil.toBean(pcaResultStr, FePcaRuleResultDto.class);

//            fePcaRuleResultDto.insertNull(overLine);

            //整合数据集
            for (int i = 0; i < data.size(); i++) {
                data.get(i).put("X_pca", JSONUtil.parseArray(fePcaRuleResultDto.getX_pca().get(i)).get(0).toString());
            }

            //添加字段表信息
            QueryWrapper<FeDatafield> feDatafieldQueryWrapper2 = new QueryWrapper<>();
            feDatafieldQueryWrapper2.in("id", datafieldIds.keySet());
            List<FeDatafield> oldFields2 = feDatafieldMapper.selectList(feDatafieldQueryWrapper2);
            List<FeDatafield> oldFields = new ArrayList<>();
            List<FeDatafield> newFields = new ArrayList<>();
            if (oldFields2.size() != 0) {
                QueryWrapper<FeDatafield> feDatafieldQueryWrapper = new QueryWrapper<>();
                feDatafieldQueryWrapper.in("datafile_id", oldFields2.get(0).getDatafileId());
                oldFields = feDatafieldMapper.selectList(feDatafieldQueryWrapper);
            }

            boolean exist = false;
            for (FeDatafield old : oldFields) {
                if (old.getName().equals("X_pca")) {
                    exist = true;
                    break;
                }
            }
            if (!exist) {
                //不存在就添加到对应字段信息表
                FeDatafield feDatafield = new FeDatafield();
                feDatafield.setDatafileId(oldFields.get(0).getDatafileId());//任意取一条 都一样
                feDatafield.setType(DataTypeEnum.FLOAT.getCode());
                feDatafield.setName("X_pca");
                feDatafield.setCreateTime(new Date());
                feDatafield.setSource(4);
                newFields.add(feDatafield);
                feDatafieldService.saveBatch(newFields);
            }


            //装载模型
//            LinkedHashMap<String, List<Double>> modelsParams = new LinkedHashMap<>();
//            for (int i = 0; i < titles.size(); i++) {
//                modelsParams.put(titles.get(i),fePcaRuleResultDto.getProj_matrix().get(i));
//            }
//
////            存储模型数据，竖标题
//            operatorUtil.putPcaData(feOperatorExecDto.getId().toString(),modelsParams);

        }

        return data;
    }


    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
    SimpleDateFormat sdf2 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");

    //调用计算前转换类型，数组结构
    public Object convert(List<String> list, String type) {
        Object p = null;
        if (type.equals(DataTypeEnum.STRING.getCode())) {
            String[] arrays = new String[list.size()];
            arrays = list.toArray(arrays);//必须制定类型
            p = ObjectUtil.clone(arrays);//数组只能转换成单个接收
        } else if (type.equals(DataTypeEnum.INTEGER.getCode())) {
            Integer[] arrays = new Integer[list.size()];
            List<Integer> list1 = list.stream().map(Integer::parseInt).collect(Collectors.toList());
            p = ObjectUtil.clone(list1.toArray(arrays));//数组只能转换成单个接收
        } else if (type.equals(DataTypeEnum.FLOAT.getCode())) {
//            double[] arrays = new double[list.size()];
            Double[] arrays = new Double[list.size()];
            List<Double> list1 = list.stream().map(Double::parseDouble).collect(Collectors.toList());
            for (int i = 0; i < list1.size(); i++) {
                arrays[i] = list1.get(i);
            }
            p = ObjectUtil.clone(list1.toArray(arrays));//数组只能转换成单个接收
//            p = arrays;//数组只能转换成单个接收
        } else if (type.equals(DataTypeEnum.FLOAT2.getCode())) {
            double[] arrays = new double[list.size()];
            List<Double> list1 = list.stream().map(Double::parseDouble).collect(Collectors.toList());
            for (int i = 0; i < list1.size(); i++) {
                arrays[i] = list1.get(i);
            }
            p = arrays;//数组只能转换成单个接收
        } else if (type.equals(DataTypeEnum.DATE.getCode())) {
            Date[] arrays = new Date[list.size()];
            List<Date> list1 = list.stream().map(r -> {
                try {
                    return sdf.parse(r);
                } catch (ParseException e) {
                    throw new BDException("日期转换失败");
                }
            }).collect(Collectors.toList());
            p = ObjectUtil.clone(list1.toArray(arrays));//数组只能转换成单个接收
        } else if (type.equals(DataTypeEnum.DATETIME.getCode())) {
            Date[] arrays = new Date[list.size()];
            List<Date> list1 = list.stream().map(r -> {
                try {
                    return sdf2.parse(r);
                } catch (ParseException e) {
                    throw new BDException("日期转换失败");
                }
            }).collect(Collectors.toList());
            p = ObjectUtil.clone(list1.toArray(arrays));//数组只能转换成单个接收
        }
        return p;
    }

    //调用计算前转换类型，基础类型结构
    public Object convert(String object, String type) {
        Object p = null;
        if (type.equals(DataTypeEnum.STRING.getCode())) {
            p = String.valueOf(object);
        } else if (type.equals(DataTypeEnum.INTEGER.getCode())) {
            p = Integer.valueOf(object);
        } else if (type.equals(DataTypeEnum.FLOAT.getCode())) {
            p = Double.valueOf(object);
        } else if (type.equals(DataTypeEnum.DATE.getCode()) || type.equals(DataTypeEnum.DATETIME.getCode())) {
            try {
                p = sdf.parse(object);
            } catch (ParseException e) {
                throw new BDException("日期转换失败");
            }
        }
        return p;
    }


}

